<!DOCTYPE HTML>
<html>
<head>
<meta charset="UTF-8">
<title>Example 7.2: Maximum entropy distribution</title>
<link rel="canonical" href="/Users/mcgrant/Repos/CVX/examples/cvxbook/Ch07_statistical_estim/html/maxent.html">
<link rel="stylesheet" href="../../../examples.css" type="text/css">
</head>
<body>
<div id="header">
<h1>Example 7.2: Maximum entropy distribution</h1>
Jump to:&nbsp;&nbsp;&nbsp;&nbsp;
<a href="#source">Source code</a>&nbsp;&nbsp;&nbsp;&nbsp;
<a href="#output">Text output</a>
&nbsp;&nbsp;&nbsp;&nbsp;
<a href="#plots">Plots</a>
&nbsp;&nbsp;&nbsp;&nbsp;<a href="../../../index.html">Library index</a>
</div>
<div id="content">
<a id="source"></a>
<pre class="codeinput">
<span class="comment">% Section 7.2, Figures 7.2-7.3</span>
<span class="comment">% Boyd &amp; Vandenberghe, "Convex Optimization"</span>
<span class="comment">% Originally by Lieven Vandenberghe</span>
<span class="comment">% Adapted for CVX by Michael Grant 4/11/06</span>
<span class="comment">%</span>
<span class="comment">% We consider a probability distribution on 100 equidistant points in the</span>
<span class="comment">% interval [-1,1]. We impose the following prior assumptions:</span>
<span class="comment">%</span>
<span class="comment">%    -0.1 &lt;= E(X) &lt;= +0.1</span>
<span class="comment">%    +0.5 &lt;= E(X^2) &lt;= +0.6</span>
<span class="comment">%    -0.3 &lt;= E(3*X^3-2*X) &lt;= -0.2</span>
<span class="comment">%    +0.3 &lt;= Pr(X&lt;0) &lt;= 0.4</span>
<span class="comment">%</span>
<span class="comment">% Along with the constraints sum(p) == 1, p &gt;= 0, these constraints</span>
<span class="comment">% describe a polyhedron of probability distrubtions. In the first figure,</span>
<span class="comment">% the distribution that maximizes entropy is computed. In the second</span>
<span class="comment">% figure, we compute upper and lower bounds for Prob(X&lt;=a_i) for each</span>
<span class="comment">% point -1 &lt;= a_i &lt;= +1 in the distribution, as well as the maximum</span>
<span class="comment">% entropy CDF.</span>

<span class="comment">%</span>
<span class="comment">% Represent the polyhedron as follows:</span>
<span class="comment">%     A * p &lt;= b</span>
<span class="comment">%     sum( p ) == 1</span>
<span class="comment">%     p &gt;= 0</span>
<span class="comment">%</span>

n  = 100;
a  = linspace(-1,1,n);
a2 = a .^ 2;
a3 = 3 * ( a.^ 3 ) - 2 * a;
ap = +( a &lt; 0 );
A  = [ a   ; -a  ; a2 ; -a2  ; a3 ; -a3 ; ap ; -ap ];
b  = [ 0.1 ; 0.1 ;0.5 ; -0.5 ; -0.2 ; 0.3 ; 0.4 ; -0.3 ];

<span class="comment">%</span>
<span class="comment">% Compute the maximum entropy distribution</span>
<span class="comment">%</span>

cvx_expert <span class="string">true</span>
cvx_begin
    variables <span class="string">pent(n)</span>
    maximize( sum(entr(pent)) )
    sum(pent) == 1;
    A * pent &lt;= b;
cvx_end

<span class="comment">%</span>
<span class="comment">% Compute the bounds on Prob(X&lt;=a_i), i=1,...,n</span>
<span class="comment">%</span>

Ubnds = zeros(1,n);
Lbnds = zeros(1,n);
<span class="keyword">for</span> t = 1 : n,
    cvx_begin <span class="string">quiet</span>
        variable <span class="string">p( n )</span>
        minimize <span class="string">sum( p(1:t) )</span>
        p &gt;= 0;
        sum( p ) == 1;
        A * p &lt;= b;
    cvx_end
    Lbnds(t) = cvx_optval;
    cvx_begin <span class="string">quiet</span>
        variable <span class="string">p( n )</span>
        maximize <span class="string">sum( p(1:t) )</span>
        p &gt;= 0;
        sum( p ) == 1;
        A * p &lt;= b;
    cvx_end
    Ubnds(t) = cvx_optval;
    disp( sprintf( <span class="string">'%g &lt;= Prob(x&lt;=%g) &lt;= %g'</span>, Lbnds(t), a(t), Ubnds(t) ) );
<span class="keyword">end</span>

<span class="comment">%</span>
<span class="comment">% Generate the figures</span>
<span class="comment">%</span>

figure( 1 )
stairs( a, pent );
xlabel( <span class="string">'x'</span> );
ylabel( <span class="string">'PDF( x )'</span> );

figure( 2 )
stairs( a, cumsum( pent ) );
grid <span class="string">on</span>;
hold <span class="string">on</span>
d = stairs(a, Lbnds,<span class="string">'r-'</span>);  set(d,<span class="string">'Color'</span>,[0 0.5 0]);
d = stairs(a, Ubnds,<span class="string">'r-'</span>);  set(d,<span class="string">'Color'</span>,[0 0.5 0]);
d = plot([-1,-1], [Lbnds(1), Ubnds(1)],<span class="string">'r-'</span>);
set(d,<span class="string">'Color'</span>,[0 0.5 0]);
axis([-1.1 1.1 -0.1 1.1]);
xlabel( <span class="string">'x'</span> );
ylabel( <span class="string">'CDF( x )'</span> );
hold <span class="string">off</span>
</pre>
<a id="output"></a>
<pre class="codeoutput">
 
Calling Mosek 9.1.9: 308 variables, 109 equality constraints
------------------------------------------------------------

MOSEK Version 9.1.9 (Build date: 2019-11-21 11:32:15)
Copyright (c) MOSEK ApS, Denmark. WWW: mosek.com
Platform: MACOSX/64-X86

Problem
  Name                   :                 
  Objective sense        : min             
  Type                   : CONIC (conic optimization problem)
  Constraints            : 109             
  Cones                  : 100             
  Scalar variables       : 308             
  Matrix variables       : 0               
  Integer variables      : 0               

Optimizer started.
Presolve started.
Linear dependency checker started.
Linear dependency checker terminated.
Eliminator started.
Freed constraints in eliminator : 0
Eliminator terminated.
Eliminator - tries                  : 1                 time                   : 0.00            
Lin. dep.  - tries                  : 1                 time                   : 0.00            
Lin. dep.  - number                 : 0               
Presolve terminated. Time: 0.00    
Problem
  Name                   :                 
  Objective sense        : min             
  Type                   : CONIC (conic optimization problem)
  Constraints            : 109             
  Cones                  : 100             
  Scalar variables       : 308             
  Matrix variables       : 0               
  Integer variables      : 0               

Optimizer  - threads                : 8               
Optimizer  - solved problem         : the primal      
Optimizer  - Constraints            : 5
Optimizer  - Cones                  : 100
Optimizer  - Scalar variables       : 303               conic                  : 300             
Optimizer  - Semi-definite variables: 0                 scalarized             : 0               
Factor     - setup time             : 0.00              dense det. time        : 0.00            
Factor     - ML order time          : 0.00              GP order time          : 0.00            
Factor     - nonzeros before factor : 15                after factor           : 15              
Factor     - dense dim.             : 0                 flops                  : 5.06e+03        
ITE PFEAS    DFEAS    GFEAS    PRSTATUS   POBJ              DOBJ              MU       TIME  
0   8.0e+01  8.1e-01  2.1e+02  0.00e+00   8.278383991e+01   -1.290927710e+02  1.0e+00  0.00  
1   9.2e+00  9.3e-02  1.2e+01  6.35e-01   -1.258106333e+01  -4.378494190e+01  1.2e-01  0.01  
2   1.1e+00  1.2e-02  8.5e-01  5.78e-01   -7.925074971e+00  -1.308368634e+01  1.4e-02  0.01  
3   1.9e-01  1.9e-03  6.5e-02  7.63e-01   -4.920183301e+00  -5.898967691e+00  2.4e-03  0.01  
4   1.9e-02  1.9e-04  1.6e-03  1.14e+00   -4.426324685e+00  -4.514412441e+00  2.4e-04  0.01  
5   1.7e-03  1.7e-05  3.9e-05  1.05e+00   -4.389622164e+00  -4.397222024e+00  2.1e-05  0.01  
6   9.8e-05  9.9e-07  5.4e-07  1.01e+00   -4.386539903e+00  -4.386981889e+00  1.2e-06  0.01  
7   9.9e-06  1.0e-07  1.7e-08  1.00e+00   -4.386323068e+00  -4.386367836e+00  1.2e-07  0.01  
8   1.3e-06  1.3e-08  8.4e-10  1.00e+00   -4.386298489e+00  -4.386304439e+00  1.7e-08  0.01  
9   1.8e-07  1.8e-09  4.1e-11  1.00e+00   -4.386294908e+00  -4.386295698e+00  2.2e-09  0.01  
10  5.3e-08  5.3e-10  6.7e-12  1.00e+00   -4.386294501e+00  -4.386294738e+00  6.6e-10  0.01  
11  3.2e-08  3.2e-10  3.1e-12  9.99e-01   -4.386294437e+00  -4.386294581e+00  4.0e-10  0.01  
12  3.2e-08  3.2e-10  3.1e-12  1.08e+00   -4.386294436e+00  -4.386294578e+00  4.0e-10  0.01  
13  3.1e-08  3.2e-10  3.0e-12  9.85e-01   -4.386294435e+00  -4.386294575e+00  3.9e-10  0.02  
14  3.1e-08  3.2e-10  3.0e-12  9.97e-01   -4.386294435e+00  -4.386294575e+00  3.9e-10  0.02  
15  7.6e-09  7.9e-11  3.5e-13  9.99e-01   -4.386294359e+00  -4.386294394e+00  9.7e-11  0.02  
Optimizer terminated. Time: 0.02    


Interior-point solution summary
  Problem status  : PRIMAL_AND_DUAL_FEASIBLE
  Solution status : OPTIMAL
  Primal.  obj: -4.3862943588e+00   nrm: 1e+00    Viol.  con: 1e-08    var: 0e+00    cones: 0e+00  
  Dual.    obj: -4.3862943940e+00   nrm: 5e+00    Viol.  con: 0e+00    var: 6e-13    cones: 0e+00  
Optimizer summary
  Optimizer                 -                        time: 0.02    
    Interior-point          - iterations : 15        time: 0.02    
      Basis identification  -                        time: 0.00    
        Primal              - iterations : 0         time: 0.00    
        Dual                - iterations : 0         time: 0.00    
        Clean primal        - iterations : 0         time: 0.00    
        Clean dual          - iterations : 0         time: 0.00    
    Simplex                 -                        time: 0.00    
      Primal simplex        - iterations : 0         time: 0.00    
      Dual simplex          - iterations : 0         time: 0.00    
    Mixed integer           - relaxations: 0         time: 0.00    

------------------------------------------------------------
Status: Solved
Optimal value (cvx_optval): +4.38629
 
0 &lt;= Prob(x&lt;=-1) &lt;= 0.329406
0 &lt;= Prob(x&lt;=-0.979798) &lt;= 0.344777
0 &lt;= Prob(x&lt;=-0.959596) &lt;= 0.360771
0 &lt;= Prob(x&lt;=-0.939394) &lt;= 0.377365
0 &lt;= Prob(x&lt;=-0.919192) &lt;= 0.394532
0 &lt;= Prob(x&lt;=-0.89899) &lt;= 0.4
0 &lt;= Prob(x&lt;=-0.878788) &lt;= 0.4
0 &lt;= Prob(x&lt;=-0.858586) &lt;= 0.4
0 &lt;= Prob(x&lt;=-0.838384) &lt;= 0.4
0 &lt;= Prob(x&lt;=-0.818182) &lt;= 0.4
0 &lt;= Prob(x&lt;=-0.79798) &lt;= 0.4
0.0116304 &lt;= Prob(x&lt;=-0.777778) &lt;= 0.4
0.0331953 &lt;= Prob(x&lt;=-0.757576) &lt;= 0.4
0.0519157 &lt;= Prob(x&lt;=-0.737374) &lt;= 0.4
0.0701091 &lt;= Prob(x&lt;=-0.717172) &lt;= 0.4
0.0859495 &lt;= Prob(x&lt;=-0.69697) &lt;= 0.4
0.0998612 &lt;= Prob(x&lt;=-0.676768) &lt;= 0.4
0.112141 &lt;= Prob(x&lt;=-0.656566) &lt;= 0.4
0.123045 &lt;= Prob(x&lt;=-0.636364) &lt;= 0.4
0.132778 &lt;= Prob(x&lt;=-0.616162) &lt;= 0.4
0.141527 &lt;= Prob(x&lt;=-0.59596) &lt;= 0.4
0.149418 &lt;= Prob(x&lt;=-0.575758) &lt;= 0.4
0.15655 &lt;= Prob(x&lt;=-0.555556) &lt;= 0.4
0.163015 &lt;= Prob(x&lt;=-0.535354) &lt;= 0.4
0.168895 &lt;= Prob(x&lt;=-0.515152) &lt;= 0.4
0.174283 &lt;= Prob(x&lt;=-0.494949) &lt;= 0.4
0.179205 &lt;= Prob(x&lt;=-0.474747) &lt;= 0.4
0.18371 &lt;= Prob(x&lt;=-0.454545) &lt;= 0.4
0.187841 &lt;= Prob(x&lt;=-0.434343) &lt;= 0.4
0.191651 &lt;= Prob(x&lt;=-0.414141) &lt;= 0.4
0.195164 &lt;= Prob(x&lt;=-0.393939) &lt;= 0.4
0.198396 &lt;= Prob(x&lt;=-0.373737) &lt;= 0.4
0.201373 &lt;= Prob(x&lt;=-0.353535) &lt;= 0.4
0.204127 &lt;= Prob(x&lt;=-0.333333) &lt;= 0.4
0.206681 &lt;= Prob(x&lt;=-0.313131) &lt;= 0.4
0.209037 &lt;= Prob(x&lt;=-0.292929) &lt;= 0.4
0.211209 &lt;= Prob(x&lt;=-0.272727) &lt;= 0.4
0.213219 &lt;= Prob(x&lt;=-0.252525) &lt;= 0.4
0.215088 &lt;= Prob(x&lt;=-0.232323) &lt;= 0.4
0.216811 &lt;= Prob(x&lt;=-0.212121) &lt;= 0.4
0.218398 &lt;= Prob(x&lt;=-0.191919) &lt;= 0.4
0.219862 &lt;= Prob(x&lt;=-0.171717) &lt;= 0.4
0.221224 &lt;= Prob(x&lt;=-0.151515) &lt;= 0.4
0.222474 &lt;= Prob(x&lt;=-0.131313) &lt;= 0.4
0.223619 &lt;= Prob(x&lt;=-0.111111) &lt;= 0.4
0.224669 &lt;= Prob(x&lt;=-0.0909091) &lt;= 0.4
0.225643 &lt;= Prob(x&lt;=-0.0707071) &lt;= 0.4
0.22653 &lt;= Prob(x&lt;=-0.0505051) &lt;= 0.4
0.227334 &lt;= Prob(x&lt;=-0.030303) &lt;= 0.4
0.3 &lt;= Prob(x&lt;=-0.010101) &lt;= 0.4
0.3 &lt;= Prob(x&lt;=0.010101) &lt;= 0.778942
0.3 &lt;= Prob(x&lt;=0.030303) &lt;= 0.792532
0.3 &lt;= Prob(x&lt;=0.0505051) &lt;= 0.806483
0.3 &lt;= Prob(x&lt;=0.0707071) &lt;= 0.819022
0.3 &lt;= Prob(x&lt;=0.0909091) &lt;= 0.825
0.3 &lt;= Prob(x&lt;=0.111111) &lt;= 0.83125
0.3 &lt;= Prob(x&lt;=0.131313) &lt;= 0.837791
0.3 &lt;= Prob(x&lt;=0.151515) &lt;= 0.841937
0.3 &lt;= Prob(x&lt;=0.171717) &lt;= 0.845957
0.3 &lt;= Prob(x&lt;=0.191919) &lt;= 0.850137
0.3 &lt;= Prob(x&lt;=0.212121) &lt;= 0.854492
0.3 &lt;= Prob(x&lt;=0.232323) &lt;= 0.859052
0.3 &lt;= Prob(x&lt;=0.252525) &lt;= 0.863811
0.3 &lt;= Prob(x&lt;=0.272727) &lt;= 0.868817
0.3 &lt;= Prob(x&lt;=0.292929) &lt;= 0.874066
0.3 &lt;= Prob(x&lt;=0.313131) &lt;= 0.877055
0.3 &lt;= Prob(x&lt;=0.333333) &lt;= 0.880067
0.3 &lt;= Prob(x&lt;=0.353535) &lt;= 0.883272
0.300787 &lt;= Prob(x&lt;=0.373737) &lt;= 0.886687
0.307695 &lt;= Prob(x&lt;=0.393939) &lt;= 0.890333
0.314397 &lt;= Prob(x&lt;=0.414141) &lt;= 0.894234
0.320909 &lt;= Prob(x&lt;=0.434343) &lt;= 0.898418
0.327232 &lt;= Prob(x&lt;=0.454545) &lt;= 0.902981
0.333379 &lt;= Prob(x&lt;=0.474747) &lt;= 0.909013
0.339323 &lt;= Prob(x&lt;=0.494949) &lt;= 0.916606
0.345134 &lt;= Prob(x&lt;=0.515152) &lt;= 0.925292
0.350719 &lt;= Prob(x&lt;=0.535354) &lt;= 0.935184
0.356201 &lt;= Prob(x&lt;=0.555556) &lt;= 0.946304
0.361491 &lt;= Prob(x&lt;=0.575758) &lt;= 0.958921
0.366603 &lt;= Prob(x&lt;=0.59596) &lt;= 0.973265
0.371622 &lt;= Prob(x&lt;=0.616162) &lt;= 0.989508
0.387329 &lt;= Prob(x&lt;=0.636364) &lt;= 1
0.410495 &lt;= Prob(x&lt;=0.656566) &lt;= 1
0.439031 &lt;= Prob(x&lt;=0.676768) &lt;= 1
0.466372 &lt;= Prob(x&lt;=0.69697) &lt;= 1
0.492663 &lt;= Prob(x&lt;=0.717172) &lt;= 1
0.518025 &lt;= Prob(x&lt;=0.737374) &lt;= 1
0.542592 &lt;= Prob(x&lt;=0.757576) &lt;= 1
0.56651 &lt;= Prob(x&lt;=0.777778) &lt;= 1
0.589941 &lt;= Prob(x&lt;=0.79798) &lt;= 1
0.613125 &lt;= Prob(x&lt;=0.818182) &lt;= 1
0.635881 &lt;= Prob(x&lt;=0.838384) &lt;= 1
0.657609 &lt;= Prob(x&lt;=0.858586) &lt;= 1
0.678314 &lt;= Prob(x&lt;=0.878788) &lt;= 1
0.697846 &lt;= Prob(x&lt;=0.89899) &lt;= 1
0.716238 &lt;= Prob(x&lt;=0.919192) &lt;= 1
0.733536 &lt;= Prob(x&lt;=0.939394) &lt;= 1
0.74974 &lt;= Prob(x&lt;=0.959596) &lt;= 1
0.764914 &lt;= Prob(x&lt;=0.979798) &lt;= 1
1 &lt;= Prob(x&lt;=1) &lt;= 1
</pre>
<a id="plots"></a>
<div id="plotoutput">
<img src="maxent__01.png" alt=""> <img src="maxent__02.png" alt=""> 
</div>
</div>
</body>
</html>